{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71fa9577",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import copy\n",
    "from datetime import datetime, timedelta\n",
    "from scipy.sparse import csr_matrix\n",
    "from scipy.stats import spearmanr,pearsonr\n",
    "import pickle\n",
    "import os  \n",
    "import h5py\n",
    "from statistics import correlation\n",
    "import networkx as nx\n",
    "from tabulate import tabulate\n",
    "from utils import *\n",
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c03c185c",
   "metadata": {},
   "source": [
    "# 处理节点+标签+非稀疏邻接矩阵数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "0280c8b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "return_col = 'vwap'\n",
    "edge = 'close'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "53177ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open (r'final/data_train_predict/node_data_szsh.pkl', 'rb') as f:\n",
    "    node_data = pickle.load(f)\n",
    "\n",
    "with open (r'final/data_train_predict/return_data_szsh.pkl', 'rb') as f:                                         \n",
    "    return_data = pickle.load(f)\n",
    "\n",
    "with open (fr'final/graph/{edge}_szsh_nonsparse_matrix.pkl', 'rb') as f:                                         \n",
    "    edge_nonsparse_adj = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d19bb758",
   "metadata": {},
   "source": [
    "## 整了个return的去极值，其实是有问题的，以后改一下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "b7141e81",
   "metadata": {},
   "outputs": [],
   "source": [
    "return_data = return_data.apply(lambda x: x.clip(*x.quantile([0.05, 0.95])))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "274dd972",
   "metadata": {},
   "source": [
    "## node, adj, label索引一致化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "33c9dbfc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataFrame 0 index type: <class 'tuple'>, example: ('2012-01', '000001.SZ')\n",
      "DataFrame 1 index type: <class 'tuple'>, example: ('2012-01', '000001.SZ')\n",
      "DataFrame 2 index type: <class 'tuple'>, example: ('2012-01', '000001.SZ')\n"
     ]
    }
   ],
   "source": [
    "node_data_copy = node_data.copy()\n",
    "return_data_copy = return_data.copy()\n",
    "edge_nonsparse_adj_copy = edge_nonsparse_adj.copy()\n",
    "\n",
    "def standardize_index(df):\n",
    "    # 创建新的标准化索引\n",
    "    new_index = []\n",
    "    for idx in df.index:\n",
    "        if isinstance(idx, tuple):\n",
    "            # 提取元组中的两个元素\n",
    "            date_part, code_part = idx\n",
    "            \n",
    "            # 标准化日期部分（无论是Period对象还是字符串）\n",
    "            if hasattr(date_part, 'strftime'):  # 检查是否为Period对象\n",
    "                date_str = date_part.strftime('%Y-%m')\n",
    "            else:\n",
    "                date_str = date_part  # 假设已经是字符串格式\n",
    "                \n",
    "            # 创建新的标准化元组\n",
    "            new_index.append((date_str, code_part))\n",
    "    \n",
    "    # 设置新索引\n",
    "    df.index = pd.MultiIndex.from_tuples(new_index)\n",
    "    return df\n",
    "\n",
    "# 应用标准化函数到所有DataFrame\n",
    "dfs = [node_data_copy, return_data_copy, edge_nonsparse_adj_copy]\n",
    "standardized_dfs = [standardize_index(df) for df in dfs]\n",
    "\n",
    "# 验证索引是否一致\n",
    "for i, df in enumerate(standardized_dfs):\n",
    "    df.index.names = ['date', 'stock']\n",
    "    print(f\"DataFrame {i} index type: {type(df.index[0])}, example: {df.index[0]}\")\n",
    "\n",
    "# 现在可以计算交集\n",
    "all_indices = [df.index for df in standardized_dfs]\n",
    "common_index = reduce(lambda x, y: x.intersection(y), all_indices)\n",
    "\n",
    "# 应用交集到所有DataFrame\n",
    "common_dataframes = [df.loc[common_index] for df in standardized_dfs]\n",
    "\n",
    "# 如果需要，可以分别赋值回原变量名\n",
    "node_data_common, return_data_common, edge_nonsparse_adj_common = common_dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "4b6f889a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "验证通过：所有仅在列中存在的股票数据都已设为NaN\n"
     ]
    }
   ],
   "source": [
    "# 确保edge_nonsparse_adj_common不包含common的行中不存在的列\n",
    "def align_stock_indices(df):\n",
    "    # 获取MultiIndex结构\n",
    "    date_level = df.index.get_level_values(0).unique()\n",
    "    result_df = df.copy()\n",
    "    \n",
    "    # 对每个日期进行处理\n",
    "    for date in date_level:\n",
    "        # 获取当前日期的数据子集\n",
    "        date_slice = df.loc[date]\n",
    "        \n",
    "        # 获取当前日期的行索引（股票代码）\n",
    "        row_stocks = set(date_slice.index)\n",
    "        \n",
    "        # 获取列索引（股票代码）\n",
    "        col_stocks = set(df.columns)\n",
    "        \n",
    "        # 找出仅在列中存在但在行中不存在的股票\n",
    "        only_in_cols = col_stocks - row_stocks\n",
    "        \n",
    "        # 如果有仅在列中存在的股票，将它们的数据设为NaN\n",
    "        if only_in_cols:\n",
    "            for stock in only_in_cols:\n",
    "                # 将该日期下该股票的所有数据设为NaN\n",
    "                if stock in result_df.columns:  # 确保股票代码在列中存在\n",
    "                    mask = result_df.index.get_level_values(0) == date\n",
    "                    result_df.loc[mask, stock] = np.nan\n",
    "    \n",
    "    return result_df\n",
    "\n",
    "# 执行处理\n",
    "edge_nonsparse_adj_common = align_stock_indices(edge_nonsparse_adj_common)\n",
    "\n",
    "# 验证结果\n",
    "def verify_alignment(df):\n",
    "    date_level = df.index.get_level_values(0).unique()\n",
    "    for date in date_level:\n",
    "        date_slice = df.loc[date]\n",
    "        row_stocks = set(date_slice.index)\n",
    "        col_stocks = set(df.columns)\n",
    "        only_in_cols = col_stocks - row_stocks\n",
    "        \n",
    "        # 检查是否所有仅在列中存在的股票的数据都是NaN\n",
    "        for stock in only_in_cols:\n",
    "            if not df.loc[date].loc[:, stock].isna().all():\n",
    "                print(f\"错误: 日期 {date} 的股票 {stock} 有非NaN值\")\n",
    "                return False\n",
    "    \n",
    "    print(\"验证通过：所有仅在列中存在的股票数据都已设为NaN\")\n",
    "\n",
    "# 验证处理结果\n",
    "verify_alignment(edge_nonsparse_adj_common)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4be202ee",
   "metadata": {},
   "source": [
    "## 筛除相关性太低的列，并正向化相关性为负的列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "d59329eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.01242582335658175\n"
     ]
    }
   ],
   "source": [
    "equal_weight_node_data = node_data_common.mean(axis=1)\n",
    "return_col_data=(return_data_common.loc[:,return_col]).values\n",
    "corr, p_value = spearmanr(equal_weight_node_data.values,return_col_data)\n",
    "print(corr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "934f47c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the spearman corr of column dlf_h_120d: 0.009477313630066991\n",
      "the spearman corr of column dwf_h_120d: -0.009477272004754749\n",
      "the spearman corr of column tovol_180d: -0.008916641594109952\n",
      "the spearman corr of column tovol_240d: -0.008914299072559109\n"
     ]
    }
   ],
   "source": [
    "for column in node_data_common.columns:\n",
    "    corr, p_value = spearmanr(node_data_common[column].values,return_col_data)\n",
    "    if abs(corr)<0.01:\n",
    "        print(f'the spearman corr of column {column}: {corr}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "bc73071a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the spearman corr of column dlf_h_10d: 0.020529772210854693\n",
      "the spearman corr of column dlf_h_120d: 0.009477313630066991\n",
      "the spearman corr of column dlf_h_20d: 0.02264248626317505\n",
      "the spearman corr of column dlf_h_60d: 0.028230057830261012\n",
      "the spearman corr of column dwf_h_10d: -0.02053126238240816\n",
      "the spearman corr of column dwf_h_120d: -0.009477272004754749\n",
      "the spearman corr of column dwf_h_20d: -0.022650758997791582\n",
      "the spearman corr of column dwf_h_60d: -0.028232708731273853\n",
      "the spearman corr of column ivol_120d: -0.027855755631422455\n",
      "the spearman corr of column ivol_20d: -0.031213182391410847\n",
      "the spearman corr of column ivol_240d: -0.028797188973179805\n",
      "the spearman corr of column ivol_60d: -0.03058054997968027\n",
      "the spearman corr of column ivr_120d: 0.034538377553403964\n",
      "the spearman corr of column ivr_20d: 0.021076594053209864\n",
      "the spearman corr of column ivr_240d: 0.040256856028442034\n",
      "the spearman corr of column ivr_60d: 0.03341843126402402\n",
      "the spearman corr of column lnamihud_10d: 0.03379232152505474\n",
      "the spearman corr of column lnamihud_120d: 0.03016703778885878\n",
      "the spearman corr of column lnamihud_20d: 0.03297079910491076\n",
      "the spearman corr of column lnamihud_240d: 0.029859550222895742\n",
      "the spearman corr of column lnamihud_5d: 0.032972864544480646\n",
      "the spearman corr of column lnamihud_60d: 0.031749508518455996\n",
      "the spearman corr of column lnto_10d: -0.026591638508087843\n",
      "the spearman corr of column lnto_120d: -0.013269502339233194\n",
      "the spearman corr of column lnto_20d: -0.02457365478967957\n",
      "the spearman corr of column lnto_240d: -0.010418942608795771\n",
      "the spearman corr of column lnto_5d: -0.025284446171146415\n",
      "the spearman corr of column lnto_60d: -0.019671288722414408\n",
      "the spearman corr of column mom_20d_120d: 0.011033045727974757\n",
      "the spearman corr of column mom_20d_180d: 0.019744640046153374\n",
      "the spearman corr of column mom_20d_240d: 0.02377665417502273\n",
      "the spearman corr of column mom_60d_120d: 0.024800725229831774\n",
      "the spearman corr of column mom_60d_180d: 0.032197650460723856\n",
      "the spearman corr of column mom_60d_240d: 0.03272122521850735\n",
      "the spearman corr of column ret_10d: -0.01912253060251632\n",
      "the spearman corr of column ret_20d: -0.01603415453219594\n",
      "the spearman corr of column ret_5d: -0.020090919610717267\n",
      "the spearman corr of column ret_60d: -0.019863869500931615\n",
      "the spearman corr of column tovol_120d: -0.012834779915418015\n",
      "the spearman corr of column tovol_180d: -0.008916641594109952\n",
      "the spearman corr of column tovol_20d: -0.011523751956494356\n",
      "the spearman corr of column tovol_240d: -0.008914299072559109\n",
      "the spearman corr of column tovol_60d: -0.015052184567682265\n",
      "the spearman corr of column vol_120d: -0.0254126514347342\n",
      "the spearman corr of column vol_180d: -0.023773730791231518\n",
      "the spearman corr of column vol_20d: -0.022919506679895302\n",
      "the spearman corr of column vol_240d: -0.0229240198290919\n",
      "the spearman corr of column vol_60d: -0.02413609560480203\n",
      "the spearman corr of column ivol_weekly_25d: -0.01903881296082355\n",
      "the spearman corr of column ivol_weekly_36d: -0.019516417748332616\n",
      "the spearman corr of column ivol_weekly_50d: -0.01918399137053438\n",
      "the spearman corr of column ivr_weekly_25d: -0.06388815359997983\n",
      "the spearman corr of column ivr_weekly_36d: -0.056749817230681975\n",
      "the spearman corr of column ivr_weekly_50d: -0.053286868838005576\n",
      "number of factors with negative correlation: 34\n"
     ]
    }
   ],
   "source": [
    "count_negative=0\n",
    "for column in node_data_common.columns:\n",
    "    corr, p_value = spearmanr(node_data_common[column].values,return_col_data)\n",
    "    print(f'the spearman corr of column {column}: {corr}')\n",
    "    if corr < 0:\n",
    "        count_negative+=1\n",
    "\n",
    "print('number of factors with negative correlation:',count_negative)                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "d68c8d2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_data_common_filtered = node_data_common.copy(deep=True)\n",
    "\n",
    "# 计算每列与目标变量（假设最后一列是目标变量）的 Spearman 相关性\n",
    "correlations = {}\n",
    "\n",
    "for col in node_data_common_filtered.columns: \n",
    "    corr, _ = spearmanr(node_data_common_filtered[col].values,return_col_data)\n",
    "    correlations[col] = corr\n",
    "\n",
    "# 删除相关性绝对值小于 0.01 的列\n",
    "cols_to_drop = [col for col, corr in correlations.items() if abs(corr) < 0.01]\n",
    "node_data_common_filtered = node_data_common_filtered.drop(columns=cols_to_drop)\n",
    "\n",
    "# 对保留的列，如果相关性为负，则将值取反\n",
    "for col in node_data_common_filtered.columns:\n",
    "    if correlations[col] < 0:\n",
    "        node_data_common_filtered[col] = -node_data_common_filtered[col]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0c66356",
   "metadata": {},
   "source": [
    "### 等权整体spearman相关系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "5e337068",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 验证是否剩下的因子满足要求\n",
    "for column in node_data_common_filtered.columns:\n",
    "    corr, p_value = spearmanr(node_data_common_filtered[column].values,return_col_data)\n",
    "    if corr<0 or abs(corr)<0.01:\n",
    "    # if abs(corr)>0.1:\n",
    "        print(f'the spearman corr of column {column}: {corr}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "7a16c275",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.05676418308426724\n"
     ]
    }
   ],
   "source": [
    "equal_weight_node_common_filtered = node_data_common_filtered.mean(axis=1)\n",
    "corr, p_value = spearmanr(equal_weight_node_common_filtered.values,return_col_data)\n",
    "print(corr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "33359070",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 剩余因子数量\n",
    "len(node_data_common_filtered.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "138c88f5",
   "metadata": {},
   "source": [
    "### 等权分时间spearman相关系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "fdcc9478",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      spearman_mean  spearman_median  n_days\n",
      "date                                        \n",
      "2012       0.026362         0.029330      12\n",
      "2013      -0.001561        -0.044188      12\n",
      "2014       0.074818         0.025213      12\n",
      "2015       0.067681         0.116028      12\n",
      "2016       0.157474         0.206493      12\n",
      "2017       0.067793         0.056748      12\n",
      "2018       0.104266         0.081554      12\n",
      "2019      -0.041259        -0.060858      12\n",
      "2020       0.031418         0.048202      12\n",
      "2021       0.066953         0.096321      12\n",
      "2022       0.040109         0.055330      12\n",
      "2023       0.130919         0.108269      12\n",
      "2024       0.064278         0.063251      12\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame({\n",
    "    'node_value': node_data_common_filtered.mean(axis=1),\n",
    "    return_col: return_col_data\n",
    "}).reset_index()  # 将 MultiIndex 转为列\n",
    "\n",
    "# 按日期分组计算 Spearman 相关系数\n",
    "results = df.groupby('date')[['node_value',return_col]].apply(\n",
    "    lambda group: spearmanr(group['node_value'], group[return_col])[0]  # 仅取相关系数\n",
    ")\n",
    "# 将日期转换为年份\n",
    "results.index = pd.to_datetime(results.index)  # 确保索引是 datetime 类型\n",
    "yearly_corr = results.groupby(results.index.year).agg(['mean', 'median', 'count'])\n",
    "\n",
    "# 重命名列\n",
    "yearly_corr.columns = ['spearman_mean', 'spearman_median', 'n_days']\n",
    "print(yearly_corr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "110aa086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>dlf_h_10d</th>\n",
       "      <th>dlf_h_20d</th>\n",
       "      <th>dlf_h_60d</th>\n",
       "      <th>dwf_h_10d</th>\n",
       "      <th>dwf_h_20d</th>\n",
       "      <th>dwf_h_60d</th>\n",
       "      <th>ivol_120d</th>\n",
       "      <th>ivol_20d</th>\n",
       "      <th>ivol_240d</th>\n",
       "      <th>ivol_60d</th>\n",
       "      <th>...</th>\n",
       "      <th>vol_180d</th>\n",
       "      <th>vol_20d</th>\n",
       "      <th>vol_240d</th>\n",
       "      <th>vol_60d</th>\n",
       "      <th>ivol_weekly_25d</th>\n",
       "      <th>ivol_weekly_36d</th>\n",
       "      <th>ivol_weekly_50d</th>\n",
       "      <th>ivr_weekly_25d</th>\n",
       "      <th>ivr_weekly_36d</th>\n",
       "      <th>ivr_weekly_50d</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>stock</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2012-01</th>\n",
       "      <th>000001.SZ</th>\n",
       "      <td>-1.390273</td>\n",
       "      <td>-0.850299</td>\n",
       "      <td>-0.931600</td>\n",
       "      <td>-1.390125</td>\n",
       "      <td>-0.850181</td>\n",
       "      <td>-0.931356</td>\n",
       "      <td>0.353401</td>\n",
       "      <td>0.082225</td>\n",
       "      <td>0.387073</td>\n",
       "      <td>0.520953</td>\n",
       "      <td>...</td>\n",
       "      <td>0.790705</td>\n",
       "      <td>1.000362</td>\n",
       "      <td>0.805109</td>\n",
       "      <td>1.178597</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000002.SZ</th>\n",
       "      <td>-0.526122</td>\n",
       "      <td>-0.007508</td>\n",
       "      <td>-0.657584</td>\n",
       "      <td>-0.525967</td>\n",
       "      <td>-0.007379</td>\n",
       "      <td>-0.657349</td>\n",
       "      <td>0.370640</td>\n",
       "      <td>0.324321</td>\n",
       "      <td>0.685357</td>\n",
       "      <td>0.265902</td>\n",
       "      <td>...</td>\n",
       "      <td>0.755297</td>\n",
       "      <td>0.736121</td>\n",
       "      <td>0.843881</td>\n",
       "      <td>0.605104</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000063.SZ</th>\n",
       "      <td>1.918566</td>\n",
       "      <td>1.730388</td>\n",
       "      <td>0.577379</td>\n",
       "      <td>1.918742</td>\n",
       "      <td>1.730543</td>\n",
       "      <td>0.577573</td>\n",
       "      <td>-0.793038</td>\n",
       "      <td>-1.265300</td>\n",
       "      <td>-0.510407</td>\n",
       "      <td>-0.515915</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.060938</td>\n",
       "      <td>-0.438481</td>\n",
       "      <td>-0.302023</td>\n",
       "      <td>-0.083915</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000100.SZ</th>\n",
       "      <td>0.482777</td>\n",
       "      <td>-1.224943</td>\n",
       "      <td>-0.016429</td>\n",
       "      <td>0.482855</td>\n",
       "      <td>-1.224902</td>\n",
       "      <td>-0.016321</td>\n",
       "      <td>0.164150</td>\n",
       "      <td>-0.928983</td>\n",
       "      <td>-1.665916</td>\n",
       "      <td>0.032752</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.572742</td>\n",
       "      <td>-0.063906</td>\n",
       "      <td>-1.042506</td>\n",
       "      <td>0.328496</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000157.SZ</th>\n",
       "      <td>-1.862190</td>\n",
       "      <td>-1.348572</td>\n",
       "      <td>-0.405300</td>\n",
       "      <td>-1.862046</td>\n",
       "      <td>-1.348462</td>\n",
       "      <td>-0.405073</td>\n",
       "      <td>0.690280</td>\n",
       "      <td>0.025709</td>\n",
       "      <td>0.427832</td>\n",
       "      <td>0.255887</td>\n",
       "      <td>...</td>\n",
       "      <td>0.152571</td>\n",
       "      <td>-0.001384</td>\n",
       "      <td>0.063547</td>\n",
       "      <td>-0.079167</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   dlf_h_10d  dlf_h_20d  dlf_h_60d  dwf_h_10d  dwf_h_20d  \\\n",
       "date    stock                                                              \n",
       "2012-01 000001.SZ  -1.390273  -0.850299  -0.931600  -1.390125  -0.850181   \n",
       "        000002.SZ  -0.526122  -0.007508  -0.657584  -0.525967  -0.007379   \n",
       "        000063.SZ   1.918566   1.730388   0.577379   1.918742   1.730543   \n",
       "        000100.SZ   0.482777  -1.224943  -0.016429   0.482855  -1.224902   \n",
       "        000157.SZ  -1.862190  -1.348572  -0.405300  -1.862046  -1.348462   \n",
       "\n",
       "                   dwf_h_60d  ivol_120d  ivol_20d  ivol_240d  ivol_60d  ...  \\\n",
       "date    stock                                                           ...   \n",
       "2012-01 000001.SZ  -0.931356   0.353401  0.082225   0.387073  0.520953  ...   \n",
       "        000002.SZ  -0.657349   0.370640  0.324321   0.685357  0.265902  ...   \n",
       "        000063.SZ   0.577573  -0.793038 -1.265300  -0.510407 -0.515915  ...   \n",
       "        000100.SZ  -0.016321   0.164150 -0.928983  -1.665916  0.032752  ...   \n",
       "        000157.SZ  -0.405073   0.690280  0.025709   0.427832  0.255887  ...   \n",
       "\n",
       "                   vol_180d   vol_20d  vol_240d   vol_60d  ivol_weekly_25d  \\\n",
       "date    stock                                                                \n",
       "2012-01 000001.SZ  0.790705  1.000362  0.805109  1.178597             -0.0   \n",
       "        000002.SZ  0.755297  0.736121  0.843881  0.605104             -0.0   \n",
       "        000063.SZ -0.060938 -0.438481 -0.302023 -0.083915             -0.0   \n",
       "        000100.SZ -0.572742 -0.063906 -1.042506  0.328496             -0.0   \n",
       "        000157.SZ  0.152571 -0.001384  0.063547 -0.079167             -0.0   \n",
       "\n",
       "                   ivol_weekly_36d  ivol_weekly_50d  ivr_weekly_25d  \\\n",
       "date    stock                                                         \n",
       "2012-01 000001.SZ             -0.0             -0.0            -0.0   \n",
       "        000002.SZ             -0.0             -0.0            -0.0   \n",
       "        000063.SZ             -0.0             -0.0            -0.0   \n",
       "        000100.SZ             -0.0             -0.0            -0.0   \n",
       "        000157.SZ             -0.0             -0.0            -0.0   \n",
       "\n",
       "                   ivr_weekly_36d  ivr_weekly_50d  \n",
       "date    stock                                      \n",
       "2012-01 000001.SZ            -0.0            -0.0  \n",
       "        000002.SZ            -0.0            -0.0  \n",
       "        000063.SZ            -0.0            -0.0  \n",
       "        000100.SZ            -0.0            -0.0  \n",
       "        000157.SZ            -0.0            -0.0  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "node_data_common_filtered.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21b3b7fc",
   "metadata": {},
   "source": [
    "## 节点描述性统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "e03474cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>节点数量</th>\n",
       "      <th>特征维度</th>\n",
       "      <th>各特征均值</th>\n",
       "      <th>各特征标准差</th>\n",
       "      <th>各特征最小值</th>\n",
       "      <th>各特征最大值</th>\n",
       "      <th>特征稀疏度</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>dlf_h_10d</th>\n",
       "      <td>269</td>\n",
       "      <td>50</td>\n",
       "      <td>-7.924268e-17</td>\n",
       "      <td>0.99814</td>\n",
       "      <td>-2.283658</td>\n",
       "      <td>1.825441</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dlf_h_20d</th>\n",
       "      <td>269</td>\n",
       "      <td>50</td>\n",
       "      <td>-2.641423e-17</td>\n",
       "      <td>0.99814</td>\n",
       "      <td>-2.366028</td>\n",
       "      <td>1.693208</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dlf_h_60d</th>\n",
       "      <td>269</td>\n",
       "      <td>50</td>\n",
       "      <td>-7.924268e-17</td>\n",
       "      <td>0.99814</td>\n",
       "      <td>-2.833993</td>\n",
       "      <td>1.370141</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dwf_h_10d</th>\n",
       "      <td>269</td>\n",
       "      <td>50</td>\n",
       "      <td>-1.452783e-16</td>\n",
       "      <td>0.99814</td>\n",
       "      <td>-2.283096</td>\n",
       "      <td>1.826113</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dwf_h_20d</th>\n",
       "      <td>269</td>\n",
       "      <td>50</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.99814</td>\n",
       "      <td>-2.366526</td>\n",
       "      <td>1.691462</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           节点数量  特征维度         各特征均值   各特征标准差    各特征最小值    各特征最大值  特征稀疏度\n",
       "dlf_h_10d   269    50 -7.924268e-17  0.99814 -2.283658  1.825441    0.0\n",
       "dlf_h_20d   269    50 -2.641423e-17  0.99814 -2.366028  1.693208    0.0\n",
       "dlf_h_60d   269    50 -7.924268e-17  0.99814 -2.833993  1.370141    0.0\n",
       "dwf_h_10d   269    50 -1.452783e-16  0.99814 -2.283096  1.826113    0.0\n",
       "dwf_h_20d   269    50  0.000000e+00  0.99814 -2.366526  1.691462    0.0"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_node = node_data_common_filtered.loc['2024-12']\n",
    "node_stats = describe_node_features(sample_node)\n",
    "node_stats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "d87bdc61",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_stats.to_csv(r'data_description/node_stats.csv',encoding = 'gbk')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f665ab3c",
   "metadata": {},
   "source": [
    "## 数据分块保存"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af033eeb",
   "metadata": {},
   "source": [
    "### 将索引改为全局共享股票代码-索引对"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "2481c2c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'000001.SZ': 0,\n",
       " '000002.SZ': 1,\n",
       " '000063.SZ': 2,\n",
       " '000100.SZ': 3,\n",
       " '000157.SZ': 4,\n",
       " '000301.SZ': 5,\n",
       " '000338.SZ': 6,\n",
       " '000425.SZ': 7,\n",
       " '000538.SZ': 8,\n",
       " '000568.SZ': 9,\n",
       " '000596.SZ': 10,\n",
       " '000617.SZ': 11,\n",
       " '000625.SZ': 12,\n",
       " '000630.SZ': 13,\n",
       " '000651.SZ': 14,\n",
       " '000661.SZ': 15,\n",
       " '000708.SZ': 16,\n",
       " '000725.SZ': 17,\n",
       " '000768.SZ': 18,\n",
       " '000776.SZ': 19,\n",
       " '000786.SZ': 20,\n",
       " '000792.SZ': 21,\n",
       " '000800.SZ': 22,\n",
       " '000807.SZ': 23,\n",
       " '000858.SZ': 24,\n",
       " '000876.SZ': 25,\n",
       " '000895.SZ': 26,\n",
       " '000938.SZ': 27,\n",
       " '000963.SZ': 28,\n",
       " '000975.SZ': 29,\n",
       " '000977.SZ': 30,\n",
       " '000983.SZ': 31,\n",
       " '000999.SZ': 32,\n",
       " '002001.SZ': 33,\n",
       " '002007.SZ': 34,\n",
       " '002027.SZ': 35,\n",
       " '002028.SZ': 36,\n",
       " '002049.SZ': 37,\n",
       " '002050.SZ': 38,\n",
       " '002074.SZ': 39,\n",
       " '002129.SZ': 40,\n",
       " '002142.SZ': 41,\n",
       " '002179.SZ': 42,\n",
       " '002180.SZ': 43,\n",
       " '002230.SZ': 44,\n",
       " '002236.SZ': 45,\n",
       " '002241.SZ': 46,\n",
       " '002252.SZ': 47,\n",
       " '002271.SZ': 48,\n",
       " '002304.SZ': 49,\n",
       " '002311.SZ': 50,\n",
       " '002352.SZ': 51,\n",
       " '002371.SZ': 52,\n",
       " '002415.SZ': 53,\n",
       " '002422.SZ': 54,\n",
       " '002459.SZ': 55,\n",
       " '002460.SZ': 56,\n",
       " '002463.SZ': 57,\n",
       " '002466.SZ': 58,\n",
       " '002475.SZ': 59,\n",
       " '300014.SZ': 60,\n",
       " '300015.SZ': 61,\n",
       " '300033.SZ': 62,\n",
       " '300059.SZ': 63,\n",
       " '300122.SZ': 64,\n",
       " '300124.SZ': 65,\n",
       " '600000.SH': 66,\n",
       " '600009.SH': 67,\n",
       " '600010.SH': 68,\n",
       " '600011.SH': 69,\n",
       " '600015.SH': 70,\n",
       " '600016.SH': 71,\n",
       " '600018.SH': 72,\n",
       " '600019.SH': 73,\n",
       " '600026.SH': 74,\n",
       " '600027.SH': 75,\n",
       " '600028.SH': 76,\n",
       " '600029.SH': 77,\n",
       " '600030.SH': 78,\n",
       " '600031.SH': 79,\n",
       " '600036.SH': 80,\n",
       " '600039.SH': 81,\n",
       " '600048.SH': 82,\n",
       " '600050.SH': 83,\n",
       " '600061.SH': 84,\n",
       " '600066.SH': 85,\n",
       " '600085.SH': 86,\n",
       " '600089.SH': 87,\n",
       " '600104.SH': 88,\n",
       " '600111.SH': 89,\n",
       " '600115.SH': 90,\n",
       " '600150.SH': 91,\n",
       " '600160.SH': 92,\n",
       " '600161.SH': 93,\n",
       " '600176.SH': 94,\n",
       " '600183.SH': 95,\n",
       " '600188.SH': 96,\n",
       " '600196.SH': 97,\n",
       " '600219.SH': 98,\n",
       " '600233.SH': 99,\n",
       " '600276.SH': 100,\n",
       " '600309.SH': 101,\n",
       " '600332.SH': 102,\n",
       " '600346.SH': 103,\n",
       " '600362.SH': 104,\n",
       " '600377.SH': 105,\n",
       " '600406.SH': 106,\n",
       " '600415.SH': 107,\n",
       " '600426.SH': 108,\n",
       " '600436.SH': 109,\n",
       " '600438.SH': 110,\n",
       " '600460.SH': 111,\n",
       " '600482.SH': 112,\n",
       " '600489.SH': 113,\n",
       " '600519.SH': 114,\n",
       " '600547.SH': 115,\n",
       " '600570.SH': 116,\n",
       " '600584.SH': 117,\n",
       " '600585.SH': 118,\n",
       " '600588.SH': 119,\n",
       " '600600.SH': 120,\n",
       " '600660.SH': 121,\n",
       " '600674.SH': 122,\n",
       " '600690.SH': 123,\n",
       " '600741.SH': 124,\n",
       " '600745.SH': 125,\n",
       " '600760.SH': 126,\n",
       " '600795.SH': 127,\n",
       " '600803.SH': 128,\n",
       " '600809.SH': 129,\n",
       " '600845.SH': 130,\n",
       " '600875.SH': 131,\n",
       " '600886.SH': 132,\n",
       " '600887.SH': 133,\n",
       " '600893.SH': 134,\n",
       " '600900.SH': 135,\n",
       " '600999.SH': 136,\n",
       " '601006.SH': 137,\n",
       " '601009.SH': 138,\n",
       " '601088.SH': 139,\n",
       " '601111.SH': 140,\n",
       " '601117.SH': 141,\n",
       " '601166.SH': 142,\n",
       " '601169.SH': 143,\n",
       " '601186.SH': 144,\n",
       " '601288.SH': 145,\n",
       " '601318.SH': 146,\n",
       " '601328.SH': 147,\n",
       " '601390.SH': 148,\n",
       " '601398.SH': 149,\n",
       " '601600.SH': 150,\n",
       " '601601.SH': 151,\n",
       " '601607.SH': 152,\n",
       " '601618.SH': 153,\n",
       " '601628.SH': 154,\n",
       " '601668.SH': 155,\n",
       " '601688.SH': 156,\n",
       " '601699.SH': 157,\n",
       " '601766.SH': 158,\n",
       " '601788.SH': 159,\n",
       " '601808.SH': 160,\n",
       " '601818.SH': 161,\n",
       " '601857.SH': 162,\n",
       " '601872.SH': 163,\n",
       " '601877.SH': 164,\n",
       " '601888.SH': 165,\n",
       " '601898.SH': 166,\n",
       " '601899.SH': 167,\n",
       " '601919.SH': 168,\n",
       " '601939.SH': 169,\n",
       " '601988.SH': 170,\n",
       " '601989.SH': 171,\n",
       " '601998.SH': 172,\n",
       " '601377.SH': 173,\n",
       " '002493.SZ': 174,\n",
       " '601799.SH': 175,\n",
       " '002555.SZ': 176,\n",
       " '600372.SH': 177,\n",
       " '002594.SZ': 178,\n",
       " '601058.SH': 179,\n",
       " '002601.SZ': 180,\n",
       " '601901.SH': 181,\n",
       " '601633.SH': 182,\n",
       " '300274.SZ': 183,\n",
       " '601100.SH': 184,\n",
       " '601669.SH': 185,\n",
       " '002648.SZ': 186,\n",
       " '601336.SH': 187,\n",
       " '601360.SH': 188,\n",
       " '601238.SH': 189,\n",
       " '601800.SH': 190,\n",
       " '300308.SZ': 191,\n",
       " '601012.SH': 192,\n",
       " '300316.SZ': 193,\n",
       " '600515.SH': 194,\n",
       " '000408.SZ': 195,\n",
       " '300347.SZ': 196,\n",
       " '603993.SH': 197,\n",
       " '000333.SZ': 198,\n",
       " '600023.SH': 199,\n",
       " '002709.SZ': 200,\n",
       " '002714.SZ': 201,\n",
       " '601225.SH': 202,\n",
       " '603288.SH': 203,\n",
       " '603369.SH': 204,\n",
       " '603806.SH': 205,\n",
       " '603019.SH': 206,\n",
       " '300408.SZ': 207,\n",
       " '002736.SZ': 208,\n",
       " '000166.SZ': 209,\n",
       " '300413.SZ': 210,\n",
       " '300418.SZ': 211,\n",
       " '601021.SH': 212,\n",
       " '603799.SH': 213,\n",
       " '300394.SZ': 214,\n",
       " '300433.SZ': 215,\n",
       " '600958.SH': 216,\n",
       " '601689.SH': 217,\n",
       " '300442.SZ': 218,\n",
       " '300450.SZ': 219,\n",
       " '601985.SH': 220,\n",
       " '601211.SH': 221,\n",
       " '300498.SZ': 222,\n",
       " '001979.SZ': 223,\n",
       " '300502.SZ': 224,\n",
       " '601127.SH': 225,\n",
       " '600919.SH': 226,\n",
       " '603986.SH': 227,\n",
       " '002812.SZ': 228,\n",
       " '600926.SH': 229,\n",
       " '601229.SH': 230,\n",
       " '601881.SH': 231,\n",
       " '300628.SZ': 232,\n",
       " '603833.SH': 233,\n",
       " '603501.SH': 234,\n",
       " '300661.SZ': 235,\n",
       " '601878.SH': 236,\n",
       " '603260.SH': 237,\n",
       " '603659.SH': 238,\n",
       " '001965.SZ': 239,\n",
       " '002916.SZ': 240,\n",
       " '002920.SZ': 241,\n",
       " '600025.SH': 242,\n",
       " '601838.SH': 243,\n",
       " '603259.SH': 244,\n",
       " '300750.SZ': 245,\n",
       " '601066.SH': 246,\n",
       " '601138.SH': 247,\n",
       " '002938.SZ': 248,\n",
       " '300760.SZ': 249,\n",
       " '601319.SH': 250,\n",
       " '300759.SZ': 251,\n",
       " '601865.SH': 252,\n",
       " '600989.SH': 253,\n",
       " '300782.SZ': 254,\n",
       " '601236.SH': 255,\n",
       " '601698.SH': 256,\n",
       " '688008.SH': 257,\n",
       " '688009.SH': 258,\n",
       " '688012.SH': 259,\n",
       " '003816.SZ': 260,\n",
       " '688036.SH': 261,\n",
       " '601916.SH': 262,\n",
       " '688111.SH': 263,\n",
       " '601658.SH': 264,\n",
       " '601816.SH': 265,\n",
       " '603195.SH': 266,\n",
       " '688169.SH': 267,\n",
       " '688396.SH': 268}"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_stocks = node_data_common.index.get_level_values('stock').unique()\n",
    "stock_to_idx = {stock: idx for idx, stock in enumerate(all_stocks)}\n",
    "stock_to_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "0fbf6ba8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对节点\n",
    "node_data_idx = node_data_common_filtered.copy()\n",
    "node_data_idx.index = node_data_idx.index.set_levels(\n",
    "    node_data_idx.index.levels[1].map(stock_to_idx),  # 替换 stock 层级\n",
    "    level='stock'  # 指定要替换的层级名称或位置\n",
    ")\n",
    "\n",
    "# 对标签\n",
    "return_data_idx = return_data_common.copy()\n",
    "return_data_idx.index = return_data_idx.index.set_levels(\n",
    "    return_data_idx.index.levels[1].map(stock_to_idx),\n",
    "    level='stock'\n",
    ")\n",
    "\n",
    "# 对邻接矩阵\n",
    "def replace_index_and_columns(df, stock_to_idx):\n",
    "    \"\"\"替换 DataFrame 的行索引（stock）和列名为数字\"\"\"\n",
    "    # 复制 DataFrame 避免修改原数据\n",
    "    df = df.copy()\n",
    "    # 1. 替换行索引中的 stock 代码为数字\n",
    "    if isinstance(df.index, pd.MultiIndex):\n",
    "        stock_level_pos = df.index.names.index('stock')  # 或直接 level=1\n",
    "        new_stock_level = df.index.levels[stock_level_pos].map(stock_to_idx)\n",
    "        df.index = df.index.set_levels(new_stock_level, level='stock')\n",
    "    # 2. 替换列名中的股票代码为数字\n",
    "    df.columns = df.columns.map(stock_to_idx)\n",
    "    return df.sort_index().sort_index(axis=1)\n",
    "\n",
    "edge_nonsparse_adj_idx = replace_index_and_columns(edge_nonsparse_adj_common, stock_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "f177dac1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>dlf_h_10d</th>\n",
       "      <th>dlf_h_20d</th>\n",
       "      <th>dlf_h_60d</th>\n",
       "      <th>dwf_h_10d</th>\n",
       "      <th>dwf_h_20d</th>\n",
       "      <th>dwf_h_60d</th>\n",
       "      <th>ivol_120d</th>\n",
       "      <th>ivol_20d</th>\n",
       "      <th>ivol_240d</th>\n",
       "      <th>ivol_60d</th>\n",
       "      <th>...</th>\n",
       "      <th>vol_180d</th>\n",
       "      <th>vol_20d</th>\n",
       "      <th>vol_240d</th>\n",
       "      <th>vol_60d</th>\n",
       "      <th>ivol_weekly_25d</th>\n",
       "      <th>ivol_weekly_36d</th>\n",
       "      <th>ivol_weekly_50d</th>\n",
       "      <th>ivr_weekly_25d</th>\n",
       "      <th>ivr_weekly_36d</th>\n",
       "      <th>ivr_weekly_50d</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>stock</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2012-01</th>\n",
       "      <th>0</th>\n",
       "      <td>-1.390273</td>\n",
       "      <td>-0.850299</td>\n",
       "      <td>-0.931600</td>\n",
       "      <td>-1.390125</td>\n",
       "      <td>-0.850181</td>\n",
       "      <td>-0.931356</td>\n",
       "      <td>0.353401</td>\n",
       "      <td>0.082225</td>\n",
       "      <td>0.387073</td>\n",
       "      <td>0.520953</td>\n",
       "      <td>...</td>\n",
       "      <td>0.790705</td>\n",
       "      <td>1.000362</td>\n",
       "      <td>0.805109</td>\n",
       "      <td>1.178597</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.526122</td>\n",
       "      <td>-0.007508</td>\n",
       "      <td>-0.657584</td>\n",
       "      <td>-0.525967</td>\n",
       "      <td>-0.007379</td>\n",
       "      <td>-0.657349</td>\n",
       "      <td>0.370640</td>\n",
       "      <td>0.324321</td>\n",
       "      <td>0.685357</td>\n",
       "      <td>0.265902</td>\n",
       "      <td>...</td>\n",
       "      <td>0.755297</td>\n",
       "      <td>0.736121</td>\n",
       "      <td>0.843881</td>\n",
       "      <td>0.605104</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.918566</td>\n",
       "      <td>1.730388</td>\n",
       "      <td>0.577379</td>\n",
       "      <td>1.918742</td>\n",
       "      <td>1.730543</td>\n",
       "      <td>0.577573</td>\n",
       "      <td>-0.793038</td>\n",
       "      <td>-1.265300</td>\n",
       "      <td>-0.510407</td>\n",
       "      <td>-0.515915</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.060938</td>\n",
       "      <td>-0.438481</td>\n",
       "      <td>-0.302023</td>\n",
       "      <td>-0.083915</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.482777</td>\n",
       "      <td>-1.224943</td>\n",
       "      <td>-0.016429</td>\n",
       "      <td>0.482855</td>\n",
       "      <td>-1.224902</td>\n",
       "      <td>-0.016321</td>\n",
       "      <td>0.164150</td>\n",
       "      <td>-0.928983</td>\n",
       "      <td>-1.665916</td>\n",
       "      <td>0.032752</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.572742</td>\n",
       "      <td>-0.063906</td>\n",
       "      <td>-1.042506</td>\n",
       "      <td>0.328496</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.862190</td>\n",
       "      <td>-1.348572</td>\n",
       "      <td>-0.405300</td>\n",
       "      <td>-1.862046</td>\n",
       "      <td>-1.348462</td>\n",
       "      <td>-0.405073</td>\n",
       "      <td>0.690280</td>\n",
       "      <td>0.025709</td>\n",
       "      <td>0.427832</td>\n",
       "      <td>0.255887</td>\n",
       "      <td>...</td>\n",
       "      <td>0.152571</td>\n",
       "      <td>-0.001384</td>\n",
       "      <td>0.063547</td>\n",
       "      <td>-0.079167</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "      <td>-0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               dlf_h_10d  dlf_h_20d  dlf_h_60d  dwf_h_10d  dwf_h_20d  \\\n",
       "date    stock                                                          \n",
       "2012-01 0      -1.390273  -0.850299  -0.931600  -1.390125  -0.850181   \n",
       "        1      -0.526122  -0.007508  -0.657584  -0.525967  -0.007379   \n",
       "        2       1.918566   1.730388   0.577379   1.918742   1.730543   \n",
       "        3       0.482777  -1.224943  -0.016429   0.482855  -1.224902   \n",
       "        4      -1.862190  -1.348572  -0.405300  -1.862046  -1.348462   \n",
       "\n",
       "               dwf_h_60d  ivol_120d  ivol_20d  ivol_240d  ivol_60d  ...  \\\n",
       "date    stock                                                       ...   \n",
       "2012-01 0      -0.931356   0.353401  0.082225   0.387073  0.520953  ...   \n",
       "        1      -0.657349   0.370640  0.324321   0.685357  0.265902  ...   \n",
       "        2       0.577573  -0.793038 -1.265300  -0.510407 -0.515915  ...   \n",
       "        3      -0.016321   0.164150 -0.928983  -1.665916  0.032752  ...   \n",
       "        4      -0.405073   0.690280  0.025709   0.427832  0.255887  ...   \n",
       "\n",
       "               vol_180d   vol_20d  vol_240d   vol_60d  ivol_weekly_25d  \\\n",
       "date    stock                                                            \n",
       "2012-01 0      0.790705  1.000362  0.805109  1.178597             -0.0   \n",
       "        1      0.755297  0.736121  0.843881  0.605104             -0.0   \n",
       "        2     -0.060938 -0.438481 -0.302023 -0.083915             -0.0   \n",
       "        3     -0.572742 -0.063906 -1.042506  0.328496             -0.0   \n",
       "        4      0.152571 -0.001384  0.063547 -0.079167             -0.0   \n",
       "\n",
       "               ivol_weekly_36d  ivol_weekly_50d  ivr_weekly_25d  \\\n",
       "date    stock                                                     \n",
       "2012-01 0                 -0.0             -0.0            -0.0   \n",
       "        1                 -0.0             -0.0            -0.0   \n",
       "        2                 -0.0             -0.0            -0.0   \n",
       "        3                 -0.0             -0.0            -0.0   \n",
       "        4                 -0.0             -0.0            -0.0   \n",
       "\n",
       "               ivr_weekly_36d  ivr_weekly_50d  \n",
       "date    stock                                  \n",
       "2012-01 0                -0.0            -0.0  \n",
       "        1                -0.0            -0.0  \n",
       "        2                -0.0            -0.0  \n",
       "        3                -0.0            -0.0  \n",
       "        4                -0.0            -0.0  \n",
       "\n",
       "[5 rows x 50 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>close</th>\n",
       "      <th>vwap</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>stock</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2012-01</th>\n",
       "      <th>0</th>\n",
       "      <td>0.039505</td>\n",
       "      <td>0.032745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.095230</td>\n",
       "      <td>0.090321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.161918</td>\n",
       "      <td>0.160837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.132305</td>\n",
       "      <td>0.146033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.102188</td>\n",
       "      <td>0.091924</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  close      vwap\n",
       "date    stock                    \n",
       "2012-01 0      0.039505  0.032745\n",
       "        1      0.095230  0.090321\n",
       "        2      0.161918  0.160837\n",
       "        3      0.132305  0.146033\n",
       "        4      0.102188  0.091924"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(node_data_idx.head())\n",
    "display(return_data_idx.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "d6a5d8c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "9\n",
      "10\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>259</th>\n",
       "      <th>260</th>\n",
       "      <th>261</th>\n",
       "      <th>262</th>\n",
       "      <th>263</th>\n",
       "      <th>264</th>\n",
       "      <th>265</th>\n",
       "      <th>266</th>\n",
       "      <th>267</th>\n",
       "      <th>268</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stock</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11 rows × 269 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1    2    3    4    5    6    7    8    9    ...  259  260  261  \\\n",
       "stock                                                    ...                  \n",
       "0        0    0    1    0    0    0    0    1    1    0  ...  1.0  0.0  0.0   \n",
       "1        0    0    0    0    0    0    0    0    0    0  ...  0.0  0.0  0.0   \n",
       "2        1    0    0    0    0    0    0    1    1    0  ...  1.0  0.0  0.0   \n",
       "3        0    0    0    0    1    0    0    0    0    0  ...  0.0  0.0  0.0   \n",
       "4        0    0    0    1    0    0    1    0    0    1  ...  0.0  0.0  0.0   \n",
       "5        0    0    0    0    0    0    1    0    0    1  ...  0.0  0.0  1.0   \n",
       "6        0    0    0    0    1    1    0    0    0    1  ...  0.0  0.0  1.0   \n",
       "7        1    0    1    0    0    0    0    0    1    0  ...  0.0  0.0  0.0   \n",
       "8        1    0    1    0    0    0    0    1    0    0  ...  1.0  0.0  0.0   \n",
       "9        0    0    0    0    1    1    1    0    0    0  ...  0.0  0.0  1.0   \n",
       "10       0    0    0    0    1    1    1    0    0    1  ...  0.0  0.0  0.0   \n",
       "\n",
       "       262  263  264  265  266  267  268  \n",
       "stock                                     \n",
       "0      1.0  0.0  1.0  1.0  0.0  0.0  1.0  \n",
       "1      0.0  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "2      1.0  0.0  1.0  0.0  0.0  0.0  1.0  \n",
       "3      0.0  1.0  0.0  0.0  0.0  0.0  0.0  \n",
       "4      0.0  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "5      0.0  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "6      0.0  0.0  0.0  0.0  0.0  0.0  0.0  \n",
       "7      1.0  0.0  1.0  0.0  1.0  0.0  0.0  \n",
       "8      1.0  0.0  1.0  1.0  0.0  0.0  0.0  \n",
       "9      0.0  1.0  0.0  0.0  0.0  0.0  0.0  \n",
       "10     0.0  1.0  0.0  0.0  0.0  0.0  0.0  \n",
       "\n",
       "[11 rows x 269 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 验证\n",
    "sample_stock = ['000538.SZ','000568.SZ','000596.SZ']\n",
    "for stock in sample_stock:\n",
    "    print(stock_to_idx[stock])\n",
    "edge_nonsparse_adj_idx.loc['2024-12'].dropna(axis=1,how='all').head(11)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0797fe7c",
   "metadata": {},
   "source": [
    "### 数据保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "f3270263",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open (r'stock/stock_to_idx.pkl', 'wb') as f:\n",
    "    pickle.dump(stock_to_idx,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0738e37",
   "metadata": {},
   "source": [
    "### x->y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "1057c3da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "节点-标签拼接输出示例：\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'第一个月数据：'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.00000000e+00, -1.39027268e+00, -8.50298710e-01, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  3.27451064e-02],\n",
       "       [ 1.00000000e+00, -5.26121929e-01, -7.50836314e-03, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  9.03213619e-02],\n",
       "       [ 2.00000000e+00,  1.91856596e+00,  1.73038785e+00, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  1.60837089e-01],\n",
       "       ...,\n",
       "       [ 1.70000000e+02, -2.37979892e-01, -7.59375648e-01, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  2.65474361e-02],\n",
       "       [ 1.71000000e+02, -1.68296072e+00, -8.75100222e-01, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  1.04378577e-01],\n",
       "       [ 1.72000000e+02, -9.12497294e-01, -1.07163023e+00, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  5.00635532e-02]],\n",
       "      shape=(173, 52))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "所有节点数据保存完毕\n",
      "所有close边数据保存完毕\n"
     ]
    }
   ],
   "source": [
    "# 节点，标签数据保存\n",
    "grouped_node = node_data_idx.groupby('date')\n",
    "grouped_return = return_data_idx.groupby('date')\n",
    "ret_col_name = return_col\n",
    "\n",
    "date_arrays = split_data_to_month(grouped_node, grouped_return, ret_col_name)\n",
    "save_month_data(date_arrays, edge_nonsparse_adj_idx,ret_col_name, edge)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd5e3c63",
   "metadata": {},
   "source": [
    "### x->rank(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "ce54a169",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               rank_vwap  rank_close\n",
      "date    stock                       \n",
      "2012-01 0      -1.161415   -1.201450\n",
      "        1      -0.080098    0.040048\n",
      "        2       1.261537    1.241499\n",
      "        3       1.041268    0.861039\n",
      "        4       0.020024    0.240290\n",
      "        5       0.200244    0.080097\n",
      "        6      -1.081317   -1.181426\n",
      "        7      -1.361659   -1.261523\n",
      "        8      -0.660805   -0.720870\n",
      "        9       0.180220   -0.260314\n"
     ]
    }
   ],
   "source": [
    "# 示例数据（假设 return_data 是你的 DataFrame）\n",
    "return_data_rank = return_data_idx.sort_index(level=['date', 'stock'])\n",
    "\n",
    "# 对每个日期的股票按 'close' 排序，并计算 Rank\n",
    "return_data_rank['rank_vwap'] = return_data_rank.groupby('date')['vwap'].rank(\n",
    "    method='average',  # 相同值取平均排名\n",
    "    ascending=True,     # 升序排列\n",
    ")\n",
    "return_data_rank['rank_close'] = return_data_rank.groupby('date')['close'].rank(\n",
    "    method='average',  # 相同值取平均排名\n",
    "    ascending=True,     # 升序排列\n",
    ")\n",
    "\n",
    "return_data_rank['rank_vwap'] = return_data_rank.groupby('date')['rank_vwap'].transform(zscore)\n",
    "return_data_rank['rank_close'] = return_data_rank.groupby('date')['rank_close'].transform(zscore)\n",
    "\n",
    "# 只保留标准化后的 rank 列\n",
    "return_data_rank = return_data_rank.loc[:, ['rank_vwap', 'rank_close']]\n",
    "\n",
    "print(return_data_rank.head(10))  # 查看结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "b0135fdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "节点-标签拼接输出示例：\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'第一个月数据：'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.00000000e+00, -1.39027268e+00, -8.50298710e-01, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00, -1.16141470e+00],\n",
       "       [ 1.00000000e+00, -5.26121929e-01, -7.50836314e-03, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00, -8.00975655e-02],\n",
       "       [ 2.00000000e+00,  1.91856596e+00,  1.73038785e+00, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  1.26153666e+00],\n",
       "       ...,\n",
       "       [ 1.70000000e+02, -2.37979892e-01, -7.59375648e-01, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00, -1.32160983e+00],\n",
       "       [ 1.71000000e+02, -1.68296072e+00, -8.75100222e-01, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00,  4.20512219e-01],\n",
       "       [ 1.72000000e+02, -9.12497294e-01, -1.07163023e+00, ...,\n",
       "        -0.00000000e+00, -0.00000000e+00, -8.81073220e-01]],\n",
       "      shape=(173, 52))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "所有节点数据保存完毕\n",
      "所有close边数据保存完毕\n"
     ]
    }
   ],
   "source": [
    "# 节点，标签数据保存\n",
    "grouped_node = node_data_idx.groupby('date')\n",
    "grouped_return = return_data_rank.groupby('date')\n",
    "ret_col_name = f'rank_{return_col}'\n",
    "\n",
    "date_arrays = split_data_to_month(grouped_node, grouped_return, ret_col_name)\n",
    "save_month_data(date_arrays, edge_nonsparse_adj_idx, ret_col_name, edge)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "data_load",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
